#include "tok.h"

#include <assert.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>

#include "sqlite3.h"

#include "tags.h"

#include "log/log.h"

#include "core/tokenizer.h"

typedef struct nim_tokenizer {
	sqlite3_tokenizer base;
	int configs[128]; /* flag ASCII */
} nim_tokenizer;

typedef struct nim_tokenizer_cursor {
	sqlite3_tokenizer_cursor base;
	const char *pInput; /* input we are tokenizing */
	int nBytes; /* size of the input */
	int iOffset; /* current position in pInput */
	int iToken; /* index of next token to be returned */
	char *pToken; /* storage for current token */
	int nTokenAllocated; /* space allocated to zToken buffer */
} nim_tokenizer_cursor;

// -1 as 'space', terminate others, skip
// 0 continuous token
// 1 single token
// 00-20 -1
// 21-2F 1 (2A -1)
// 30-39 0
// 3A-40 1
// 41-5A 0
// 5B-60 1
// 61-7A 0
// 7B-7E 1
// 7F -1
static void config_token(int configs[]) {
	int i = 0;
	for (i = 0x00; i <= 0x20; i++) {configs[i] = -1;}
	for (i = 0x21; i <= 0x2F; i++) {configs[i] = 1;} configs[0x2A] = -1;
	for (i = 0x30; i <= 0x39; i++) {configs[i] = 0;}
	for (i = 0x3A; i <= 0x40; i++) {configs[i] = 1;}
	for (i = 0x41; i <= 0x5A; i++) {configs[i] = 0;}
	for (i = 0x5B; i <= 0x60; i++) {configs[i] = 1;}
	for (i = 0x61; i <= 0x7A; i++) {configs[i] = 0;}
	for (i = 0x7B; i <= 0x7E; i++) {configs[i] = 1;}
	for (i = 0x7F; i <= 0x7F; i++) {configs[i] = -1;}
}

/*
** Create a new tokenizer instance.
*/
static int nimCreate(
		int argc,
		const char * const *argv,
		sqlite3_tokenizer **ppTokenizer
) {
	nim_tokenizer *t;

	t = (nim_tokenizer *) sqlite3_malloc(sizeof(*t));
	if (t == NULL)
		return SQLITE_NOMEM;
	memset(t, 0, sizeof(*t));

	config_token(t->configs);

	*ppTokenizer = &t->base;
	return SQLITE_OK;
}

/*
** Destroy a tokenizer
*/
static int nimDestroy(
		sqlite3_tokenizer *pTokenizer
) {
	sqlite3_free(pTokenizer);
	return SQLITE_OK;
}

/*
** Prepare to begin tokenizing a particular string.  The input
** string to be tokenized is pInput[0..nBytes-1].  A cursor
** used to incrementally tokenize this string is returned in 
** *ppCursor.
*/
static int nimOpen(
		sqlite3_tokenizer *pTokenizer, /* The tokenizer */
		const char *pInput,
		int nBytes, /* String to be tokenized */
		sqlite3_tokenizer_cursor **ppCursor /* OUT: Tokenization cursor */
) {
#ifdef TOKENIZER_DEBUG
	char buffer[1024] = { 0 };
	if (nBytes == -1) {
		strcpy(buffer, pInput);
	} else {
		strncpy(buffer, pInput, nBytes);
	}
	log_print(ANDROID_LOG_DEBUG, TAG_NIM_INDEX_TOKEN, "open %s", buffer);
#endif

	nim_tokenizer_cursor *c;

	c = (nim_tokenizer_cursor *) sqlite3_malloc(sizeof(*c));
	if (c == NULL)
		return SQLITE_NOMEM;

	c->pInput = pInput;
	if (pInput == 0) {
		c->nBytes = 0;
	} else if (nBytes < 0) {
		c->nBytes = (int) strlen(pInput);
	} else {
		c->nBytes = nBytes;
	}
	c->iOffset = 0; /* start tokenizing at the beginning */
	c->iToken = 0;
	c->pToken = NULL; /* no space allocated, yet. */
	c->nTokenAllocated = 0;

	*ppCursor = &c->base;
	return SQLITE_OK;
}

/*
** Close a tokenization cursor previously opened by a call to
** nimOpen() above.
*/
static int nimClose(
		sqlite3_tokenizer_cursor *pCursor
) {
#ifdef TOKENIZER_DEBUG
	log_print(ANDROID_LOG_DEBUG, TAG_NIM_INDEX_TOKEN, "close");
#endif

	nim_tokenizer_cursor *c = (nim_tokenizer_cursor *) pCursor;
	sqlite3_free(c->pToken);
	sqlite3_free(c);
	return SQLITE_OK;
}

/*
** Extract the next token from a tokenization cursor.  The cursor must
** have been opened by a prior call to nimOpen().
*/
static int nimNext(
		sqlite3_tokenizer_cursor *pCursor, /* Cursor returned by nimOpen */
		const char **ppToken, /* OUT: *ppToken is the token text */
		int *pnBytes, /* OUT: Number of bytes in token */
		int *piStartOffset, /* OUT: Starting offset of token */
		int *piEndOffset, /* OUT: Ending offset of token */
		int *piPosition /* OUT: Position integer of token */
) {
	nim_tokenizer_cursor *cursor = (nim_tokenizer_cursor *) pCursor;
	nim_tokenizer *tokenizer = (nim_tokenizer *) pCursor->pTokenizer;
	unsigned char *input = (unsigned char *) cursor->pInput;

	int iStartOffset = -1;
	while (cursor->iOffset < cursor->nBytes) {
		unsigned char c = input[cursor->iOffset];

		if (c < 0x80) {
			int config = tokenizer->configs[c];
			if (config == 0) { // continuous token
				// begin
				if (iStartOffset == -1) {
					iStartOffset = cursor->iOffset;
				}

				// next
				cursor->iOffset++;
			} else if (config > 0) { // single token
				// catch it
				if (iStartOffset == -1) {
					iStartOffset = cursor->iOffset++;
				}

				// if break continuous token, catch it in next turn
				break;
			} else { // space
				if (iStartOffset != -1) {
					// break continuous token, skip it in next turn
					break;
				} else {
					// skip
					cursor->iOffset++;
				}
			}
		} else {
			if (iStartOffset == -1) {
				iStartOffset = cursor->iOffset;

				int length = 0;
				for (; c & 0x80; c <<= 1, length++) {}

				int remain = cursor->nBytes - cursor->iOffset;
				cursor->iOffset += length > remain ? remain : length;
			}

			break;
		}
	}

	if (iStartOffset != -1) {
		int count = cursor->iOffset - iStartOffset;

		if (count > cursor->nTokenAllocated) {
			char *pNew;
			cursor->nTokenAllocated = count + 20;
			pNew = sqlite3_realloc(cursor->pToken, cursor->nTokenAllocated);
			if (!pNew)
				return SQLITE_NOMEM;
			cursor->pToken = pNew;
		}

		int i = 0;
		for (i = 0; i < count; i++) {
			/* TODO This needs expansion to handle UTF-8
			 ** case-insensitivity.
			 */
			unsigned char chr = input[iStartOffset + i];
			cursor->pToken[i] = (char) ((chr >= 'A' && chr <= 'Z') ? chr - 'A' + 'a' : chr);
		}

		*ppToken = cursor->pToken;
		*pnBytes = count;
		*piStartOffset = iStartOffset;
		*piEndOffset = cursor->iOffset;
		*piPosition = cursor->iToken++;

#ifdef TOKENIZER_DEBUG
		char buffer[1024] = { 0 };
		strncpy(buffer, *ppToken, *pnBytes);
		log_print(ANDROID_LOG_DEBUG, TAG_NIM_INDEX_TOKEN, "token: %s %d %d %d", buffer, *piStartOffset, *piEndOffset, *piPosition);
#endif

		return SQLITE_OK;
	}

	return SQLITE_DONE;
}

static const sqlite3_tokenizer_module nimTokenizerModule = {
		0,
		nimCreate,
		nimDestroy,
		nimOpen,
		nimClose,
		nimNext,
		0,
};

const sqlite3_tokenizer_module * tokenizer_nim() {
	return &nimTokenizerModule;
}
